{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cohen's Kappa Statistics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Agreement Rate Calculation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Note:** : this notebook assumes the use of **Python 3**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preamble: Settings Django Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%load preamble_directives.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Weighted Cohen's Kappa Function\n",
    "\n",
    "$kappa = 1 - \\frac{\\sum W*X}{\\sum W*M}$ where $*$ indicates the element-wise matrix multiplication.\n",
    "\n",
    "$X$: Is the matrix of Observed Scores\n",
    "\n",
    "$M$: Is the matrix of Score Agreement by Chance\n",
    "\n",
    "$W$: Is the Weight Matrix.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from django.contrib.auth.models import User\n",
    "from source_code_analysis.models import SoftwareProject"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from evaluations import Judge"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculate Agreement Score (Function Definition)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from evaluations import calculate_agreement_scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cohens' Kappa Function (definition)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from evaluations import cohens_kappa"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cohens' Kappa 3 WITHOUT logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CoffeeMaker & 1.000 & 1.000\n",
      "JFreechart (0.6.0) & 1.000 & 1.000\n",
      "JFreechart (0.7.1) & 1.000 & 1.000\n",
      "JHotDraw (7.4.1) & 0.999 & 0.999\n"
     ]
    }
   ],
   "source": [
    "j1 = Judge('leonardo.nole', 'CoffeeMaker')\n",
    "j2 = Judge('rossella.linsalata', 'CoffeeMaker')\n",
    "J = calculate_agreement_scores(j1, j2)\n",
    "unweighted_k = cohens_kappa(J)\n",
    "weighted_k = cohens_kappa(J, weighted=True)\n",
    "print('CoffeeMaker & %.3f & %.3f' % (unweighted_k, weighted_k))\n",
    "\n",
    "j1 = Judge('leonardo.nole', 'JFreechart', '0.6.0')\n",
    "j2 = Judge('antonio.petrone', 'JFreechart', '0.6.0')\n",
    "J = calculate_agreement_scores(j1, j2)\n",
    "unweighted_k = cohens_kappa(J)\n",
    "weighted_k = cohens_kappa(J, weighted=True)\n",
    "print('JFreechart (0.6.0) & %.3f & %.3f' % (unweighted_k, weighted_k))\n",
    "\n",
    "j1 = Judge('leonardo.nole', 'JFreechart', '0.7.1')\n",
    "j2 = Judge('antonio.petrone', 'JFreechart', '0.7.1')\n",
    "J = calculate_agreement_scores(j1, j2)\n",
    "unweighted_k = cohens_kappa(J)\n",
    "weighted_k = cohens_kappa(J, weighted=True)\n",
    "print('JFreechart (0.7.1) & %.3f & %.3f' % (unweighted_k, weighted_k))\n",
    "\n",
    "j1 = Judge('leonardo.nole', 'JHotDraw', '7.4.1')\n",
    "j2 = Judge('rossella.linsalata', 'JHotDraw', '7.4.1')\n",
    "J = calculate_agreement_scores(j1, j2)\n",
    "unweighted_k = cohens_kappa(J)\n",
    "weighted_k = cohens_kappa(J, weighted=True)\n",
    "print('JHotDraw (7.4.1) & %.3f & %.3f' % (unweighted_k, weighted_k))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cohen's Kappa 3 WITH logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------------------------------------\n",
      "\t\t CoffeeMaker\n",
      "--------------------------------------------------------------------------------\n",
      "J: \n",
      " [[19  0  2]\n",
      " [ 0  0  0]\n",
      " [ 0  0 26]]\n",
      "X: \n",
      " [[ 0.40425532  0.          0.04255319]\n",
      " [ 0.          0.          0.        ]\n",
      " [ 0.          0.          0.55319149]]\n",
      "W: \n",
      " [[ 0.  1.  1.]\n",
      " [ 1.  0.  1.]\n",
      " [ 1.  1.  0.]]\n",
      "AG: \n",
      " [[ 21.   0.  26.]\n",
      " [ 19.   0.  28.]]\n",
      "J_sum: \n",
      " [[ 0.44680851  0.          0.55319149]\n",
      " [ 0.40425532  0.          0.59574468]]\n",
      "M: \n",
      " [[ 0.18062472  0.          0.26618379]\n",
      " [ 0.          0.          0.        ]\n",
      " [ 0.2236306   0.          0.32956089]]\n",
      "J: \n",
      " [[19  0  2]\n",
      " [ 0  0  0]\n",
      " [ 0  0 26]]\n",
      "X: \n",
      " [[ 0.40425532  0.          0.04255319]\n",
      " [ 0.          0.          0.        ]\n",
      " [ 0.          0.          0.55319149]]\n",
      "W: \n",
      " [[ 0.  1.  4.]\n",
      " [ 1.  0.  1.]\n",
      " [ 4.  1.  0.]]\n",
      "AG: \n",
      " [[ 21.   0.  26.]\n",
      " [ 19.   0.  28.]]\n",
      "J_sum: \n",
      " [[ 0.44680851  0.          0.55319149]\n",
      " [ 0.40425532  0.          0.59574468]]\n",
      "M: \n",
      " [[ 0.18062472  0.          0.26618379]\n",
      " [ 0.          0.          0.        ]\n",
      " [ 0.2236306   0.          0.32956089]]\n",
      "Kappa: 0.913 & 0.913\n",
      "--------------------------------------------------------------------------------\n",
      "\t\t JfreeChart 0.6.0\n",
      "--------------------------------------------------------------------------------\n",
      "J: \n",
      " [[ 47   0   0]\n",
      " [  0  24   0]\n",
      " [  8   0 406]]\n",
      "X: \n",
      " [[ 0.09690722  0.          0.        ]\n",
      " [ 0.          0.04948454  0.        ]\n",
      " [ 0.01649485  0.          0.8371134 ]]\n",
      "W: \n",
      " [[ 0.  1.  1.]\n",
      " [ 1.  0.  1.]\n",
      " [ 1.  1.  0.]]\n",
      "AG: \n",
      " [[  47.   24.  414.]\n",
      " [  55.   24.  406.]]\n",
      "J_sum: \n",
      " [[ 0.09690722  0.04948454  0.85360825]\n",
      " [ 0.11340206  0.04948454  0.8371134 ]]\n",
      "M: \n",
      " [[ 0.01098948  0.00479541  0.08112233]\n",
      " [ 0.00561165  0.00244872  0.04142417]\n",
      " [ 0.09680094  0.04224041  0.7145669 ]]\n",
      "J: \n",
      " [[ 47   0   0]\n",
      " [  0  24   0]\n",
      " [  8   0 406]]\n",
      "X: \n",
      " [[ 0.09690722  0.          0.        ]\n",
      " [ 0.          0.04948454  0.        ]\n",
      " [ 0.01649485  0.          0.8371134 ]]\n",
      "W: \n",
      " [[ 0.  1.  4.]\n",
      " [ 1.  0.  1.]\n",
      " [ 4.  1.  0.]]\n",
      "AG: \n",
      " [[  47.   24.  414.]\n",
      " [  55.   24.  406.]]\n",
      "J_sum: \n",
      " [[ 0.09690722  0.04948454  0.85360825]\n",
      " [ 0.11340206  0.04948454  0.8371134 ]]\n",
      "M: \n",
      " [[ 0.01098948  0.00479541  0.08112233]\n",
      " [ 0.00561165  0.00244872  0.04142417]\n",
      " [ 0.09680094  0.04224041  0.7145669 ]]\n",
      "Kappa: 0.939 & 0.918\n",
      "--------------------------------------------------------------------------------\n",
      "\t\t JfreeChart 0.7.1\n",
      "--------------------------------------------------------------------------------\n",
      "J: \n",
      " [[ 65   0   0]\n",
      " [  0  36   0]\n",
      " [  3   0 520]]\n",
      "X: \n",
      " [[ 0.10416667  0.          0.        ]\n",
      " [ 0.          0.05769231  0.        ]\n",
      " [ 0.00480769  0.          0.83333333]]\n",
      "W: \n",
      " [[ 0.  1.  1.]\n",
      " [ 1.  0.  1.]\n",
      " [ 1.  1.  0.]]\n",
      "AG: \n",
      " [[  65.   36.  523.]\n",
      " [  68.   36.  520.]]\n",
      "J_sum: \n",
      " [[ 0.10416667  0.05769231  0.83814103]\n",
      " [ 0.10897436  0.05769231  0.83333333]]\n",
      "M: \n",
      " [[ 0.0113515   0.00600962  0.08680556]\n",
      " [ 0.00628698  0.0033284   0.04807692]\n",
      " [ 0.09133588  0.04835429  0.69845085]]\n",
      "J: \n",
      " [[ 65   0   0]\n",
      " [  0  36   0]\n",
      " [  3   0 520]]\n",
      "X: \n",
      " [[ 0.10416667  0.          0.        ]\n",
      " [ 0.          0.05769231  0.        ]\n",
      " [ 0.00480769  0.          0.83333333]]\n",
      "W: \n",
      " [[ 0.  1.  4.]\n",
      " [ 1.  0.  1.]\n",
      " [ 4.  1.  0.]]\n",
      "AG: \n",
      " [[  65.   36.  523.]\n",
      " [  68.   36.  520.]]\n",
      "J_sum: \n",
      " [[ 0.10416667  0.05769231  0.83814103]\n",
      " [ 0.10897436  0.05769231  0.83333333]]\n",
      "M: \n",
      " [[ 0.0113515   0.00600962  0.08680556]\n",
      " [ 0.00628698  0.0033284   0.04807692]\n",
      " [ 0.09133588  0.04835429  0.69845085]]\n",
      "Kappa: 0.983 & 0.977\n",
      "--------------------------------------------------------------------------------\n",
      "\t\t JHotDraw\n",
      "--------------------------------------------------------------------------------\n",
      "J: \n",
      " [[808   0   0]\n",
      " [  0 676   0]\n",
      " [289   2 705]]\n",
      "X: \n",
      " [[ 0.32580645  0.          0.        ]\n",
      " [ 0.          0.27258065  0.        ]\n",
      " [ 0.11653226  0.00080645  0.28427419]]\n",
      "W: \n",
      " [[ 0.  1.  1.]\n",
      " [ 1.  0.  1.]\n",
      " [ 1.  1.  0.]]\n",
      "AG: \n",
      " [[  808.   676.   996.]\n",
      " [ 1097.   678.   705.]]\n",
      "J_sum: \n",
      " [[ 0.32580645  0.27258065  0.4016129 ]\n",
      " [ 0.44233871  0.2733871   0.28427419]]\n",
      "M: \n",
      " [[ 0.14411681  0.08907128  0.09261837]\n",
      " [ 0.12057297  0.07452003  0.07748764]\n",
      " [ 0.17764893  0.10979579  0.11416818]]\n",
      "J: \n",
      " [[808   0   0]\n",
      " [  0 676   0]\n",
      " [289   2 705]]\n",
      "X: \n",
      " [[ 0.32580645  0.          0.        ]\n",
      " [ 0.          0.27258065  0.        ]\n",
      " [ 0.11653226  0.00080645  0.28427419]]\n",
      "W: \n",
      " [[ 0.  1.  4.]\n",
      " [ 1.  0.  1.]\n",
      " [ 4.  1.  0.]]\n",
      "AG: \n",
      " [[  808.   676.   996.]\n",
      " [ 1097.   678.   705.]]\n",
      "J_sum: \n",
      " [[ 0.32580645  0.27258065  0.4016129 ]\n",
      " [ 0.44233871  0.2733871   0.28427419]]\n",
      "M: \n",
      " [[ 0.14411681  0.08907128  0.09261837]\n",
      " [ 0.12057297  0.07452003  0.07748764]\n",
      " [ 0.17764893  0.10979579  0.11416818]]\n",
      "Kappa: 0.824 & 0.684\n"
     ]
    }
   ],
   "source": [
    "print('-'*80)\n",
    "print('\\t\\t CoffeeMaker')\n",
    "print('-'*80)\n",
    "j1 = Judge('leonardo.nole', 'CoffeeMaker')\n",
    "j2 = Judge('rossella.linsalata', 'CoffeeMaker')\n",
    "J = calculate_agreement_scores(j1, j2)\n",
    "unweighted_k = cohens_kappa(J, log=True)\n",
    "weighted_k = cohens_kappa(J, weighted=True, log=True)\n",
    "print('Kappa: %.3f & %.3f' % (unweighted_k, weighted_k))\n",
    "\n",
    "print('-'*80)\n",
    "print('\\t\\t JfreeChart 0.6.0')\n",
    "print('-'*80)\n",
    "j1 = Judge('leonardo.nole', 'JFreechart', '0.6.0')\n",
    "j2 = Judge('antonio.petrone', 'JFreechart', '0.6.0')\n",
    "J = calculate_agreement_scores(j1, j2)\n",
    "unweighted_k = cohens_kappa(J, log=True)\n",
    "weighted_k = cohens_kappa(J, weighted=True, log=True)\n",
    "print('Kappa: %.3f & %.3f' % (unweighted_k, weighted_k))\n",
    "\n",
    "print('-'*80)\n",
    "print('\\t\\t JfreeChart 0.7.1')\n",
    "print('-'*80)\n",
    "j1 = Judge('leonardo.nole', 'JFreechart', '0.7.1')\n",
    "j2 = Judge('antonio.petrone', 'JFreechart', '0.7.1')\n",
    "J = calculate_agreement_scores(j1, j2)\n",
    "unweighted_k = cohens_kappa(J, log=True)\n",
    "weighted_k = cohens_kappa(J, weighted=True, log=True)\n",
    "print('Kappa: %.3f & %.3f' % (unweighted_k, weighted_k))\n",
    "\n",
    "print('-'*80)\n",
    "print('\\t\\t JHotDraw')\n",
    "print('-'*80)\n",
    "j1 = Judge('leonardo.nole', 'JHotDraw', '7.4.1')\n",
    "j2 = Judge('rossella.linsalata', 'JHotDraw', '7.4.1')\n",
    "J = calculate_agreement_scores(j1, j2)\n",
    "unweighted_k = cohens_kappa(J, log=True)\n",
    "weighted_k = cohens_kappa(J, weighted=True, log=True)\n",
    "print('Kappa: %.3f & %.3f' % (unweighted_k, weighted_k))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cohen's Kappa 5 WITHOUT logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CoffeeMaker & 0.282 & 0.807\n",
      "JFreechart (0.6.0) & 0.202 & 0.657\n",
      "JFreechart (0.7.1) & 0.163 & 0.669\n",
      "JHotDraw (7.4.1) & 0.585 & 0.564\n"
     ]
    }
   ],
   "source": [
    "j1 = Judge('leonardo.nole', 'CoffeeMaker')\n",
    "j2 = Judge('rossella.linsalata', 'CoffeeMaker')\n",
    "J = calculate_agreement_scores(j1, j2, k=5)\n",
    "unweighted_k = cohens_kappa(J)\n",
    "weighted_k = cohens_kappa(J, weighted=True)\n",
    "print('CoffeeMaker & %.3f & %.3f' % (unweighted_k, weighted_k))\n",
    "\n",
    "j1 = Judge('leonardo.nole', 'JFreechart', '0.6.0')\n",
    "j2 = Judge('antonio.petrone', 'JFreechart', '0.6.0')\n",
    "J = calculate_agreement_scores(j1, j2, k=5)\n",
    "unweighted_k = cohens_kappa(J)\n",
    "weighted_k = cohens_kappa(J, weighted=True)\n",
    "print('JFreechart (0.6.0) & %.3f & %.3f' % (unweighted_k, weighted_k))\n",
    "\n",
    "j1 = Judge('leonardo.nole', 'JFreechart', '0.7.1')\n",
    "j2 = Judge('antonio.petrone', 'JFreechart', '0.7.1')\n",
    "J = calculate_agreement_scores(j1, j2, k=5)\n",
    "unweighted_k = cohens_kappa(J)\n",
    "weighted_k = cohens_kappa(J, weighted=True)\n",
    "print('JFreechart (0.7.1) & %.3f & %.3f' % (unweighted_k, weighted_k))\n",
    "\n",
    "j1 = Judge('leonardo.nole', 'JHotDraw', '7.4.1')\n",
    "j2 = Judge('rossella.linsalata', 'JHotDraw', '7.4.1')\n",
    "J = calculate_agreement_scores(j1, j2, k=5)\n",
    "unweighted_k = cohens_kappa(J)\n",
    "weighted_k = cohens_kappa(J, weighted=True)\n",
    "print('JHotDraw (7.4.1) & %.3f & %.3f' % (unweighted_k, weighted_k))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cohen's Kappa 5 WITH logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------------------------------------\n",
      "\t\t CoffeeMaker\n",
      "--------------------------------------------------------------------------------\n",
      "J: \n",
      " [[ 1  5  0  1  0]\n",
      " [ 0 13  0  0  1]\n",
      " [ 0  0  0  0  0]\n",
      " [ 0  0  0  0  5]\n",
      " [ 0  0  0 12  9]]\n",
      "X: \n",
      " [[ 0.0212766   0.10638298  0.          0.0212766   0.        ]\n",
      " [ 0.          0.27659574  0.          0.          0.0212766 ]\n",
      " [ 0.          0.          0.          0.          0.        ]\n",
      " [ 0.          0.          0.          0.          0.10638298]\n",
      " [ 0.          0.          0.          0.25531915  0.19148936]]\n",
      "W: \n",
      " [[ 0.  1.  1.  1.  1.]\n",
      " [ 1.  0.  1.  1.  1.]\n",
      " [ 1.  1.  0.  1.  1.]\n",
      " [ 1.  1.  1.  0.  1.]\n",
      " [ 1.  1.  1.  1.  0.]]\n",
      "AG: \n",
      " [[  7.  14.   0.   5.  21.]\n",
      " [  1.  18.   0.  13.  15.]]\n",
      "J_sum: \n",
      " [[ 0.14893617  0.29787234  0.          0.10638298  0.44680851]\n",
      " [ 0.0212766   0.38297872  0.          0.27659574  0.31914894]]\n",
      "M: \n",
      " [[ 0.00316885  0.05703938  0.          0.04119511  0.04753282]\n",
      " [ 0.00633771  0.11407877  0.          0.08239022  0.09506564]\n",
      " [ 0.          0.          0.          0.          0.        ]\n",
      " [ 0.00226347  0.04074242  0.          0.02942508  0.03395201]\n",
      " [ 0.00950656  0.17111815  0.          0.12358533  0.14259846]]\n",
      "J: \n",
      " [[ 1  5  0  1  0]\n",
      " [ 0 13  0  0  1]\n",
      " [ 0  0  0  0  0]\n",
      " [ 0  0  0  0  5]\n",
      " [ 0  0  0 12  9]]\n",
      "X: \n",
      " [[ 0.0212766   0.10638298  0.          0.0212766   0.        ]\n",
      " [ 0.          0.27659574  0.          0.          0.0212766 ]\n",
      " [ 0.          0.          0.          0.          0.        ]\n",
      " [ 0.          0.          0.          0.          0.10638298]\n",
      " [ 0.          0.          0.          0.25531915  0.19148936]]\n",
      "W: \n",
      " [[  0.   1.   4.   9.  16.]\n",
      " [  1.   0.   1.   4.   9.]\n",
      " [  4.   1.   0.   1.   4.]\n",
      " [  9.   4.   1.   0.   1.]\n",
      " [ 16.   9.   4.   1.   0.]]\n",
      "AG: \n",
      " [[  7.  14.   0.   5.  21.]\n",
      " [  1.  18.   0.  13.  15.]]\n",
      "J_sum: \n",
      " [[ 0.14893617  0.29787234  0.          0.10638298  0.44680851]\n",
      " [ 0.0212766   0.38297872  0.          0.27659574  0.31914894]]\n",
      "M: \n",
      " [[ 0.00316885  0.05703938  0.          0.04119511  0.04753282]\n",
      " [ 0.00633771  0.11407877  0.          0.08239022  0.09506564]\n",
      " [ 0.          0.          0.          0.          0.        ]\n",
      " [ 0.00226347  0.04074242  0.          0.02942508  0.03395201]\n",
      " [ 0.00950656  0.17111815  0.          0.12358533  0.14259846]]\n",
      "Kappa: 0.282 & 0.807\n",
      "--------------------------------------------------------------------------------\n",
      "\t\t JFreechart 0.6.0\n",
      "--------------------------------------------------------------------------------\n",
      "J: \n",
      " [[  9  17   0   0   0]\n",
      " [  0  21   0   0   0]\n",
      " [  0   0  24   0   0]\n",
      " [  0   2   0  96   1]\n",
      " [  0   6   0 282  27]]\n",
      "X: \n",
      " [[ 0.0185567   0.03505155  0.          0.          0.        ]\n",
      " [ 0.          0.04329897  0.          0.          0.        ]\n",
      " [ 0.          0.          0.04948454  0.          0.        ]\n",
      " [ 0.          0.00412371  0.          0.19793814  0.00206186]\n",
      " [ 0.          0.01237113  0.          0.5814433   0.0556701 ]]\n",
      "W: \n",
      " [[ 0.  1.  1.  1.  1.]\n",
      " [ 1.  0.  1.  1.  1.]\n",
      " [ 1.  1.  0.  1.  1.]\n",
      " [ 1.  1.  1.  0.  1.]\n",
      " [ 1.  1.  1.  1.  0.]]\n",
      "AG: \n",
      " [[  26.   21.   24.   99.  315.]\n",
      " [   9.   46.   24.  378.   28.]]\n",
      "J_sum: \n",
      " [[ 0.05360825  0.04329897  0.04948454  0.20412371  0.64948454]\n",
      " [ 0.0185567   0.09484536  0.04948454  0.77938144  0.05773196]]\n",
      "M: \n",
      " [[ 0.00099479  0.00508449  0.00265278  0.04178127  0.00309491]\n",
      " [ 0.00080349  0.00410671  0.00214263  0.03374641  0.00249973]\n",
      " [ 0.00091827  0.00469338  0.00244872  0.03856733  0.00285684]\n",
      " [ 0.00378786  0.01936019  0.01010097  0.15909023  0.01178446]\n",
      " [ 0.01205229  0.0616006   0.03213944  0.5061962   0.03749601]]\n",
      "J: \n",
      " [[  9  17   0   0   0]\n",
      " [  0  21   0   0   0]\n",
      " [  0   0  24   0   0]\n",
      " [  0   2   0  96   1]\n",
      " [  0   6   0 282  27]]\n",
      "X: \n",
      " [[ 0.0185567   0.03505155  0.          0.          0.        ]\n",
      " [ 0.          0.04329897  0.          0.          0.        ]\n",
      " [ 0.          0.          0.04948454  0.          0.        ]\n",
      " [ 0.          0.00412371  0.          0.19793814  0.00206186]\n",
      " [ 0.          0.01237113  0.          0.5814433   0.0556701 ]]\n",
      "W: \n",
      " [[  0.   1.   4.   9.  16.]\n",
      " [  1.   0.   1.   4.   9.]\n",
      " [  4.   1.   0.   1.   4.]\n",
      " [  9.   4.   1.   0.   1.]\n",
      " [ 16.   9.   4.   1.   0.]]\n",
      "AG: \n",
      " [[  26.   21.   24.   99.  315.]\n",
      " [   9.   46.   24.  378.   28.]]\n",
      "J_sum: \n",
      " [[ 0.05360825  0.04329897  0.04948454  0.20412371  0.64948454]\n",
      " [ 0.0185567   0.09484536  0.04948454  0.77938144  0.05773196]]\n",
      "M: \n",
      " [[ 0.00099479  0.00508449  0.00265278  0.04178127  0.00309491]\n",
      " [ 0.00080349  0.00410671  0.00214263  0.03374641  0.00249973]\n",
      " [ 0.00091827  0.00469338  0.00244872  0.03856733  0.00285684]\n",
      " [ 0.00378786  0.01936019  0.01010097  0.15909023  0.01178446]\n",
      " [ 0.01205229  0.0616006   0.03213944  0.5061962   0.03749601]]\n",
      "Kappa: 0.202 & 0.657\n",
      "--------------------------------------------------------------------------------\n",
      "\t\t JFreechart 0.7.1\n",
      "--------------------------------------------------------------------------------\n",
      "J: \n",
      " [[ 11  15   0   0   0]\n",
      " [  4  35   0   0   0]\n",
      " [  0   0  36   0   0]\n",
      " [  0   3   0  65   0]\n",
      " [  0   0   0 440  15]]\n",
      "X: \n",
      " [[ 0.01762821  0.02403846  0.          0.          0.        ]\n",
      " [ 0.00641026  0.05608974  0.          0.          0.        ]\n",
      " [ 0.          0.          0.05769231  0.          0.        ]\n",
      " [ 0.          0.00480769  0.          0.10416667  0.        ]\n",
      " [ 0.          0.          0.          0.70512821  0.02403846]]\n",
      "W: \n",
      " [[ 0.  1.  1.  1.  1.]\n",
      " [ 1.  0.  1.  1.  1.]\n",
      " [ 1.  1.  0.  1.  1.]\n",
      " [ 1.  1.  1.  0.  1.]\n",
      " [ 1.  1.  1.  1.  0.]]\n",
      "AG: \n",
      " [[  26.   39.   36.   68.  455.]\n",
      " [  15.   53.   36.  505.   15.]]\n",
      "J_sum: \n",
      " [[ 0.04166667  0.0625      0.05769231  0.10897436  0.72916667]\n",
      " [ 0.02403846  0.0849359   0.05769231  0.80929487  0.02403846]]\n",
      "M: \n",
      " [[ 0.0010016   0.003539    0.00240385  0.03372062  0.0010016 ]\n",
      " [ 0.0015024   0.00530849  0.00360577  0.05058093  0.0015024 ]\n",
      " [ 0.00138683  0.00490015  0.0033284   0.04669009  0.00138683]\n",
      " [ 0.00261958  0.00925583  0.00628698  0.08819239  0.00261958]\n",
      " [ 0.01752804  0.06193243  0.04206731  0.59011084  0.01752804]]\n",
      "J: \n",
      " [[ 11  15   0   0   0]\n",
      " [  4  35   0   0   0]\n",
      " [  0   0  36   0   0]\n",
      " [  0   3   0  65   0]\n",
      " [  0   0   0 440  15]]\n",
      "X: \n",
      " [[ 0.01762821  0.02403846  0.          0.          0.        ]\n",
      " [ 0.00641026  0.05608974  0.          0.          0.        ]\n",
      " [ 0.          0.          0.05769231  0.          0.        ]\n",
      " [ 0.          0.00480769  0.          0.10416667  0.        ]\n",
      " [ 0.          0.          0.          0.70512821  0.02403846]]\n",
      "W: \n",
      " [[  0.   1.   4.   9.  16.]\n",
      " [  1.   0.   1.   4.   9.]\n",
      " [  4.   1.   0.   1.   4.]\n",
      " [  9.   4.   1.   0.   1.]\n",
      " [ 16.   9.   4.   1.   0.]]\n",
      "AG: \n",
      " [[  26.   39.   36.   68.  455.]\n",
      " [  15.   53.   36.  505.   15.]]\n",
      "J_sum: \n",
      " [[ 0.04166667  0.0625      0.05769231  0.10897436  0.72916667]\n",
      " [ 0.02403846  0.0849359   0.05769231  0.80929487  0.02403846]]\n",
      "M: \n",
      " [[ 0.0010016   0.003539    0.00240385  0.03372062  0.0010016 ]\n",
      " [ 0.0015024   0.00530849  0.00360577  0.05058093  0.0015024 ]\n",
      " [ 0.00138683  0.00490015  0.0033284   0.04669009  0.00138683]\n",
      " [ 0.00261958  0.00925583  0.00628698  0.08819239  0.00261958]\n",
      " [ 0.01752804  0.06193243  0.04206731  0.59011084  0.01752804]]\n",
      "Kappa: 0.163 & 0.669\n",
      "--------------------------------------------------------------------------------\n",
      "\t\t JHotDraw\n",
      "--------------------------------------------------------------------------------\n",
      "J: \n",
      " [[ 70  45   0   0   0]\n",
      " [ 15 678   0   0   0]\n",
      " [  0   0 676   0   0]\n",
      " [  0  35   1 235   1]\n",
      " [  0 254   1 448  21]]\n",
      "X: \n",
      " [[ 0.02822581  0.01814516  0.          0.          0.        ]\n",
      " [ 0.00604839  0.2733871   0.          0.          0.        ]\n",
      " [ 0.          0.          0.27258065  0.          0.        ]\n",
      " [ 0.          0.0141129   0.00040323  0.09475806  0.00040323]\n",
      " [ 0.          0.10241935  0.00040323  0.18064516  0.00846774]]\n",
      "W: \n",
      " [[ 0.  1.  1.  1.  1.]\n",
      " [ 1.  0.  1.  1.  1.]\n",
      " [ 1.  1.  0.  1.  1.]\n",
      " [ 1.  1.  1.  0.  1.]\n",
      " [ 1.  1.  1.  1.  0.]]\n",
      "AG: \n",
      " [[  115.   693.   676.   272.   724.]\n",
      " [   85.  1012.   678.   683.    22.]]\n",
      "J_sum: \n",
      " [[ 0.04637097  0.27943548  0.27258065  0.10967742  0.29193548]\n",
      " [ 0.03427419  0.40806452  0.2733871   0.27540323  0.00887097]]\n",
      "M: \n",
      " [[ 0.00158933  0.01892235  0.01267722  0.01277071  0.00041136]\n",
      " [ 0.00957743  0.11402771  0.07639406  0.07695743  0.00247886]\n",
      " [ 0.00934248  0.11123049  0.07452003  0.07506959  0.00241805]\n",
      " [ 0.00375911  0.04475546  0.02998439  0.03020552  0.00097294]\n",
      " [ 0.01000585  0.11912851  0.07981139  0.08039997  0.00258975]]\n",
      "J: \n",
      " [[ 70  45   0   0   0]\n",
      " [ 15 678   0   0   0]\n",
      " [  0   0 676   0   0]\n",
      " [  0  35   1 235   1]\n",
      " [  0 254   1 448  21]]\n",
      "X: \n",
      " [[ 0.02822581  0.01814516  0.          0.          0.        ]\n",
      " [ 0.00604839  0.2733871   0.          0.          0.        ]\n",
      " [ 0.          0.          0.27258065  0.          0.        ]\n",
      " [ 0.          0.0141129   0.00040323  0.09475806  0.00040323]\n",
      " [ 0.          0.10241935  0.00040323  0.18064516  0.00846774]]\n",
      "W: \n",
      " [[  0.   1.   4.   9.  16.]\n",
      " [  1.   0.   1.   4.   9.]\n",
      " [  4.   1.   0.   1.   4.]\n",
      " [  9.   4.   1.   0.   1.]\n",
      " [ 16.   9.   4.   1.   0.]]\n",
      "AG: \n",
      " [[  115.   693.   676.   272.   724.]\n",
      " [   85.  1012.   678.   683.    22.]]\n",
      "J_sum: \n",
      " [[ 0.04637097  0.27943548  0.27258065  0.10967742  0.29193548]\n",
      " [ 0.03427419  0.40806452  0.2733871   0.27540323  0.00887097]]\n",
      "M: \n",
      " [[ 0.00158933  0.01892235  0.01267722  0.01277071  0.00041136]\n",
      " [ 0.00957743  0.11402771  0.07639406  0.07695743  0.00247886]\n",
      " [ 0.00934248  0.11123049  0.07452003  0.07506959  0.00241805]\n",
      " [ 0.00375911  0.04475546  0.02998439  0.03020552  0.00097294]\n",
      " [ 0.01000585  0.11912851  0.07981139  0.08039997  0.00258975]]\n",
      "Kappa: 0.585 & 0.564\n"
     ]
    }
   ],
   "source": [
    "print('-'*80)\n",
    "print('\\t\\t CoffeeMaker')\n",
    "print('-'*80)\n",
    "j1 = Judge('leonardo.nole', 'CoffeeMaker')\n",
    "j2 = Judge('rossella.linsalata', 'CoffeeMaker')\n",
    "J = calculate_agreement_scores(j1, j2, k=5)\n",
    "unweighted_k = cohens_kappa(J, log=True)\n",
    "weighted_k = cohens_kappa(J, weighted=True, log=True)\n",
    "print('Kappa: %.3f & %.3f' % (unweighted_k, weighted_k))\n",
    "\n",
    "print('-'*80)\n",
    "print('\\t\\t JFreechart 0.6.0')\n",
    "print('-'*80)\n",
    "j1 = Judge('leonardo.nole', 'JFreechart', '0.6.0')\n",
    "j2 = Judge('antonio.petrone', 'JFreechart', '0.6.0')\n",
    "J = calculate_agreement_scores(j1, j2, k=5)\n",
    "unweighted_k = cohens_kappa(J, log=True)\n",
    "weighted_k = cohens_kappa(J, weighted=True, log=True)\n",
    "print('Kappa: %.3f & %.3f' % (unweighted_k, weighted_k))\n",
    "\n",
    "print('-'*80)\n",
    "print('\\t\\t JFreechart 0.7.1')\n",
    "print('-'*80)\n",
    "j1 = Judge('leonardo.nole', 'JFreechart', '0.7.1')\n",
    "j2 = Judge('antonio.petrone', 'JFreechart', '0.7.1')\n",
    "J = calculate_agreement_scores(j1, j2, k=5)\n",
    "unweighted_k = cohens_kappa(J, log=True)\n",
    "weighted_k = cohens_kappa(J, weighted=True, log=True)\n",
    "print('Kappa: %.3f & %.3f' % (unweighted_k, weighted_k))\n",
    "\n",
    "print('-'*80)\n",
    "print('\\t\\t JHotDraw')\n",
    "print('-'*80)\n",
    "j1 = Judge('leonardo.nole', 'JHotDraw', '7.4.1')\n",
    "j2 = Judge('rossella.linsalata', 'JHotDraw', '7.4.1')\n",
    "J = calculate_agreement_scores(j1, j2, k=5)\n",
    "unweighted_k = cohens_kappa(J, log=True)\n",
    "weighted_k = cohens_kappa(J, weighted=True, log=True)\n",
    "print('Kappa: %.3f & %.3f' % (unweighted_k, weighted_k))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cohens' Kappa 2 WITHOUT logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CoffeeMaker & 0.913\n",
      "JFreechart (0.6.0) & 0.912\n",
      "JFreechart (0.7.1) & 0.975\n",
      "JHotDraw (7.4.1) & 0.686\n"
     ]
    }
   ],
   "source": [
    "# ------\n",
    "# NOTE: In this case Weighted and Unweighted are exactly the same\n",
    "# ------\n",
    "\n",
    "j1 = Judge('leonardo.nole', 'CoffeeMaker')\n",
    "j2 = Judge('rossella.linsalata', 'CoffeeMaker')\n",
    "J = calculate_agreement_scores(j1, j2, k=2)\n",
    "unweighted_k = cohens_kappa(J)\n",
    "# weighted_k = cohens_kappa(J, weighted=True)\n",
    "print('CoffeeMaker & %.3f' % (unweighted_k))\n",
    "\n",
    "j1 = Judge('leonardo.nole', 'JFreechart', '0.6.0')\n",
    "j2 = Judge('antonio.petrone', 'JFreechart', '0.6.0')\n",
    "J = calculate_agreement_scores(j1, j2, k=2)\n",
    "unweighted_k = cohens_kappa(J)\n",
    "# weighted_k = cohens_kappa(J, weighted=True)\n",
    "print('JFreechart (0.6.0) & %.3f' % (unweighted_k))\n",
    "\n",
    "j1 = Judge('leonardo.nole', 'JFreechart', '0.7.1')\n",
    "j2 = Judge('antonio.petrone', 'JFreechart', '0.7.1')\n",
    "J = calculate_agreement_scores(j1, j2, k=2)\n",
    "unweighted_k = cohens_kappa(J)\n",
    "# weighted_k = cohens_kappa(J, weighted=True)\n",
    "print('JFreechart (0.7.1) & %.3f' % (unweighted_k,))\n",
    "\n",
    "j1 = Judge('leonardo.nole', 'JHotDraw', '7.4.1')\n",
    "j2 = Judge('rossella.linsalata', 'JHotDraw', '7.4.1')\n",
    "J = calculate_agreement_scores(j1, j2, k=2)\n",
    "unweighted_k = cohens_kappa(J)\n",
    "# weighted_k = cohens_kappa(J, weighted=True)\n",
    "print('JHotDraw (7.4.1) & %.3f' % (unweighted_k))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cohen's Kappa 2 WITH logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "J: \n",
      " [[19  2]\n",
      " [ 0 26]]\n",
      "X: \n",
      " [[ 0.40425532  0.04255319]\n",
      " [ 0.          0.55319149]]\n",
      "W: \n",
      " [[ 0.  1.]\n",
      " [ 1.  0.]]\n",
      "AG: \n",
      " [[ 21.  26.]\n",
      " [ 19.  28.]]\n",
      "J_sum: \n",
      " [[ 0.44680851  0.55319149]\n",
      " [ 0.40425532  0.59574468]]\n",
      "M: \n",
      " [[ 0.18062472  0.26618379]\n",
      " [ 0.2236306   0.32956089]]\n",
      "CoffeeMaker & 0.913\n",
      "J: \n",
      " [[ 47   0]\n",
      " [  8 406]]\n",
      "X: \n",
      " [[ 0.10195228  0.        ]\n",
      " [ 0.01735358  0.88069414]]\n",
      "W: \n",
      " [[ 0.  1.]\n",
      " [ 1.  0.]]\n",
      "AG: \n",
      " [[  47.  414.]\n",
      " [  55.  406.]]\n",
      "J_sum: \n",
      " [[ 0.10195228  0.89804772]\n",
      " [ 0.11930586  0.88069414]]\n",
      "M: \n",
      " [[ 0.0121635   0.08978877]\n",
      " [ 0.10714235  0.79090537]]\n",
      "JFreechart (0.6.0) & 0.912\n",
      "J: \n",
      " [[ 65   0]\n",
      " [  3 520]]\n",
      "X: \n",
      " [[ 0.11054422  0.        ]\n",
      " [ 0.00510204  0.88435374]]\n",
      "W: \n",
      " [[ 0.  1.]\n",
      " [ 1.  0.]]\n",
      "AG: \n",
      " [[  65.  523.]\n",
      " [  68.  520.]]\n",
      "J_sum: \n",
      " [[ 0.11054422  0.88945578]\n",
      " [ 0.11564626  0.88435374]]\n",
      "M: \n",
      " [[ 0.01278403  0.09776019]\n",
      " [ 0.10286223  0.78659355]]\n",
      "JFreechart (0.7.1) & 0.975\n",
      "J: \n",
      " [[808   0]\n",
      " [289 705]]\n",
      "X: \n",
      " [[ 0.44839068  0.        ]\n",
      " [ 0.16037736  0.39123196]]\n",
      "W: \n",
      " [[ 0.  1.]\n",
      " [ 1.  0.]]\n",
      "AG: \n",
      " [[  808.   994.]\n",
      " [ 1097.   705.]]\n",
      "J_sum: \n",
      " [[ 0.44839068  0.55160932]\n",
      " [ 0.60876804  0.39123196]]\n",
      "M: \n",
      " [[ 0.27296591  0.17542477]\n",
      " [ 0.33580212  0.2158072 ]]\n",
      "JHotDraw (7.4.1) & 0.686\n"
     ]
    }
   ],
   "source": [
    "# ------\n",
    "# NOTE: In this case Weighted and Unweighted are exactly the same\n",
    "# ------\n",
    "\n",
    "j1 = Judge('leonardo.nole', 'CoffeeMaker')\n",
    "j2 = Judge('rossella.linsalata', 'CoffeeMaker')\n",
    "J = calculate_agreement_scores(j1, j2, k=2)\n",
    "unweighted_k = cohens_kappa(J, log=True)\n",
    "# weighted_k = cohens_kappa(J, weighted=True)\n",
    "print('CoffeeMaker & %.3f' % (unweighted_k))\n",
    "\n",
    "j1 = Judge('leonardo.nole', 'JFreechart', '0.6.0')\n",
    "j2 = Judge('antonio.petrone', 'JFreechart', '0.6.0')\n",
    "J = calculate_agreement_scores(j1, j2, k=2)\n",
    "unweighted_k = cohens_kappa(J, log=True)\n",
    "# weighted_k = cohens_kappa(J, weighted=True)\n",
    "print('JFreechart (0.6.0) & %.3f' % (unweighted_k))\n",
    "\n",
    "j1 = Judge('leonardo.nole', 'JFreechart', '0.7.1')\n",
    "j2 = Judge('antonio.petrone', 'JFreechart', '0.7.1')\n",
    "J = calculate_agreement_scores(j1, j2, k=2)\n",
    "unweighted_k = cohens_kappa(J, log=True)\n",
    "# weighted_k = cohens_kappa(J, weighted=True)\n",
    "print('JFreechart (0.7.1) & %.3f' % (unweighted_k,))\n",
    "\n",
    "j1 = Judge('leonardo.nole', 'JHotDraw', '7.4.1')\n",
    "j2 = Judge('rossella.linsalata', 'JHotDraw', '7.4.1')\n",
    "J = calculate_agreement_scores(j1, j2, k=2)\n",
    "unweighted_k = cohens_kappa(J, log=True)\n",
    "# weighted_k = cohens_kappa(J, weighted=True)\n",
    "print('JHotDraw (7.4.1) & %.3f' % (unweighted_k))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculate the Mean Precision of Judges' evaluations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from evaluations import mean_precision"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Mean Precision \"Coherent\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CoffeeMaker & 1.000 & 0.929 & 0.963\n",
      "JFreeChart (0.6.0) & 0.981 & 1.000 & 0.990\n",
      "JFreeChart (0.7.1) & 0.994 & 1.000 & 0.997\n",
      "JHotDraw (7.4.1) & 0.708 & 1.000 & 0.829\n"
     ]
    }
   ],
   "source": [
    "j1 = Judge('leonardo.nole', 'CoffeeMaker')\n",
    "j2 = Judge('rossella.linsalata', 'CoffeeMaker')\n",
    "j1_eval = j1.three_codes_evaluations[2]\n",
    "j2_eval = j2.three_codes_evaluations[2]\n",
    "pj1, pj2, f = mean_precision(j1_eval, j2_eval)\n",
    "print('CoffeeMaker & %.3f & %.3f & %.3f' % (pj1, pj2, f))\n",
    "\n",
    "j1 = Judge('leonardo.nole', 'JFreechart', '0.6.0')\n",
    "j2 = Judge('antonio.petrone', 'JFreechart', '0.6.0')\n",
    "j1_eval = j1.three_codes_evaluations[2]\n",
    "j2_eval = j2.three_codes_evaluations[2]\n",
    "pj1, pj2, f = mean_precision(j1_eval, j2_eval)\n",
    "print('JFreeChart (0.6.0) & %.3f & %.3f & %.3f' % (pj1, pj2, f))\n",
    "\n",
    "j1 = Judge('leonardo.nole', 'JFreechart', '0.7.1')\n",
    "j2 = Judge('antonio.petrone', 'JFreechart', '0.7.1')\n",
    "j1_eval = j1.three_codes_evaluations[2]\n",
    "j2_eval = j2.three_codes_evaluations[2]\n",
    "pj1, pj2, f = mean_precision(j1_eval, j2_eval)\n",
    "print('JFreeChart (0.7.1) & %.3f & %.3f & %.3f' % (pj1, pj2, f))\n",
    "\n",
    "j1 = Judge('leonardo.nole', 'JHotDraw', '7.4.1')\n",
    "j2 = Judge('rossella.linsalata', 'JHotDraw', '7.4.1')\n",
    "j1_eval = j1.three_codes_evaluations[2]\n",
    "j2_eval = j2.three_codes_evaluations[2]\n",
    "pj1, pj2, f = mean_precision(j1_eval, j2_eval)\n",
    "print('JHotDraw (7.4.1) & %.3f & %.3f & %.3f' % (pj1, pj2, f))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Mean Precision \"Non Coherent\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CoffeeMaker & 0.905 & 1.000 & 0.950\n",
      "JFreeChart (0.6.0) & 1.000 & 0.855 & 0.922\n",
      "JFreeChart (0.7.1) & 1.000 & 0.956 & 0.977\n",
      "JHotDraw (7.4.1) & 1.000 & 0.737 & 0.848\n"
     ]
    }
   ],
   "source": [
    "j1 = Judge('leonardo.nole', 'CoffeeMaker')\n",
    "j2 = Judge('rossella.linsalata', 'CoffeeMaker')\n",
    "j1_eval = j1.three_codes_evaluations[0]\n",
    "j2_eval = j2.three_codes_evaluations[0]\n",
    "pj1, pj2, f = mean_precision(j1_eval, j2_eval)\n",
    "print('CoffeeMaker & %.3f & %.3f & %.3f' % (pj1, pj2, f))\n",
    "\n",
    "j1 = Judge('leonardo.nole', 'JFreechart', '0.6.0')\n",
    "j2 = Judge('antonio.petrone', 'JFreechart', '0.6.0')\n",
    "j1_eval = j1.three_codes_evaluations[0]\n",
    "j2_eval = j2.three_codes_evaluations[0]\n",
    "pj1, pj2, f = mean_precision(j1_eval, j2_eval)\n",
    "print('JFreeChart (0.6.0) & %.3f & %.3f & %.3f' % (pj1, pj2, f))\n",
    "\n",
    "j1 = Judge('leonardo.nole', 'JFreechart', '0.7.1')\n",
    "j2 = Judge('antonio.petrone', 'JFreechart', '0.7.1')\n",
    "j1_eval = j1.three_codes_evaluations[0]\n",
    "j2_eval = j2.three_codes_evaluations[0]\n",
    "pj1, pj2, f = mean_precision(j1_eval, j2_eval)\n",
    "print('JFreeChart (0.7.1) & %.3f & %.3f & %.3f' % (pj1, pj2, f))\n",
    "\n",
    "j1 = Judge('leonardo.nole', 'JHotDraw', '7.4.1')\n",
    "j2 = Judge('rossella.linsalata', 'JHotDraw', '7.4.1')\n",
    "j1_eval = j1.three_codes_evaluations[0]\n",
    "j2_eval = j2.three_codes_evaluations[0]\n",
    "pj1, pj2, f = mean_precision(j1_eval, j2_eval)\n",
    "print('JHotDraw (7.4.1) & %.3f & %.3f & %.3f' % (pj1, pj2, f))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check the Differences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from source_code_analysis.models import AgreementEvaluation, SoftwareProject\n",
    "from django.contrib.auth.models import User"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### CoffeeMaker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "J1: \n",
      "J2: \n",
      "J1: \n",
      "J2: \n"
     ]
    }
   ],
   "source": [
    "j1 = Judge('leonardo.nole', 'CoffeeMaker')\n",
    "j2 = Judge('rossella.linsalata', 'CoffeeMaker')\n",
    "\n",
    "j1_evals = j1.two_codes_evaluations\n",
    "j2_evals = j2.two_codes_evaluations\n",
    "\n",
    "neg_diff = j1_evals[0].intersection(j2_evals[1])\n",
    "pos_diff = j1_evals[1].intersection(j2_evals[0])\n",
    "\n",
    "leo = User.objects.get(username='leonardo.nole')\n",
    "ros = User.objects.get(username='rossella.linsalata')\n",
    "\n",
    "# -------------------------\n",
    "# NEG\n",
    "# -------------------------\n",
    "\n",
    "neg_id_list = list()\n",
    "for meth_id in neg_diff:\n",
    "    ag_eval = AgreementEvaluation.objects.get(reference_method__id=meth_id, evaluator=leo)\n",
    "    neg_id_list.append(str(ag_eval.pk))\n",
    "print('J1:', ','.join(neg_id_list))\n",
    "\n",
    "neg_id_list = list()\n",
    "for meth_id in neg_diff:\n",
    "    ag_eval = AgreementEvaluation.objects.get(reference_method__id=meth_id, evaluator=ros)\n",
    "    neg_id_list.append(str(ag_eval.pk))\n",
    "print('J2:', ','.join(neg_id_list))\n",
    "\n",
    "# -------------------------\n",
    "# POS\n",
    "# -------------------------\n",
    "pos_id_list = list()\n",
    "for meth_id in pos_diff:\n",
    "    ag_eval = AgreementEvaluation.objects.get(reference_method__id=meth_id, evaluator=leo)\n",
    "    pos_id_list.append(str(ag_eval.pk))\n",
    "print('J1:', ','.join(pos_id_list))\n",
    "\n",
    "pos_id_list = list()\n",
    "for meth_id in pos_diff:\n",
    "    ag_eval = AgreementEvaluation.objects.get(reference_method__id=meth_id, evaluator=ros)\n",
    "    pos_id_list.append(str(ag_eval.pk))\n",
    "print('J2:', ','.join(pos_id_list))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### JFreeChart 0.6.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "J1: \n",
      "J2: \n",
      "J1: \n",
      "J2: \n"
     ]
    }
   ],
   "source": [
    "j1 = Judge('leonardo.nole', 'JFreechart', '0.6.0')\n",
    "j2 = Judge('antonio.petrone', 'JFreechart', '0.6.0')\n",
    "\n",
    "j1_evals = j1.two_codes_evaluations\n",
    "j2_evals = j2.two_codes_evaluations\n",
    "\n",
    "neg_diff = j1_evals[0].intersection(j2_evals[1])\n",
    "pos_diff = j1_evals[1].intersection(j2_evals[0])\n",
    "\n",
    "leo = User.objects.get(username='leonardo.nole')\n",
    "anto = User.objects.get(username='antonio.petrone')\n",
    "\n",
    "# -------------------------\n",
    "# NEG\n",
    "# -------------------------\n",
    "\n",
    "neg_id_list = list()\n",
    "for meth_id in neg_diff:\n",
    "    ag_eval = AgreementEvaluation.objects.get(reference_method__id=meth_id, evaluator=leo)\n",
    "    neg_id_list.append(str(ag_eval.pk))\n",
    "print('J1:', ','.join(neg_id_list))\n",
    "\n",
    "neg_id_list = list()\n",
    "for meth_id in neg_diff:\n",
    "    ag_eval = AgreementEvaluation.objects.get(reference_method__id=meth_id, evaluator=anto)\n",
    "    neg_id_list.append(str(ag_eval.pk))\n",
    "print('J2:', ','.join(neg_id_list))\n",
    "\n",
    "# -------------------------\n",
    "# POS\n",
    "# -------------------------\n",
    "pos_id_list = list()\n",
    "for meth_id in pos_diff:\n",
    "    ag_eval = AgreementEvaluation.objects.get(reference_method__id=meth_id, evaluator=leo)\n",
    "    pos_id_list.append(str(ag_eval.pk))\n",
    "print('J1:', ','.join(pos_id_list))\n",
    "\n",
    "pos_id_list = list()\n",
    "for meth_id in pos_diff:\n",
    "    ag_eval = AgreementEvaluation.objects.get(reference_method__id=meth_id, evaluator=anto)\n",
    "    pos_id_list.append(str(ag_eval.pk))\n",
    "print('J2:', ','.join(pos_id_list))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### JFreeChart 0.7.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "J1: \n",
      "J2: \n",
      "J1: \n",
      "J2: \n"
     ]
    }
   ],
   "source": [
    "j1 = Judge('leonardo.nole', 'JFreechart', '0.7.1')\n",
    "j2 = Judge('antonio.petrone', 'JFreechart', '0.7.1')\n",
    "\n",
    "j1_evals = j1.two_codes_evaluations\n",
    "j2_evals = j2.two_codes_evaluations\n",
    "\n",
    "neg_diff = j1_evals[0].intersection(j2_evals[1])\n",
    "pos_diff = j1_evals[1].intersection(j2_evals[0])\n",
    "\n",
    "leo = User.objects.get(username='leonardo.nole')\n",
    "anto = User.objects.get(username='antonio.petrone')\n",
    "\n",
    "# -------------------------\n",
    "# NEG\n",
    "# -------------------------\n",
    "\n",
    "neg_id_list = list()\n",
    "for meth_id in neg_diff:\n",
    "    ag_eval = AgreementEvaluation.objects.get(reference_method__id=meth_id, evaluator=leo)\n",
    "    neg_id_list.append(str(ag_eval.pk))\n",
    "print('J1:', ','.join(neg_id_list))\n",
    "\n",
    "neg_id_list = list()\n",
    "for meth_id in neg_diff:\n",
    "    ag_eval = AgreementEvaluation.objects.get(reference_method__id=meth_id, evaluator=anto)\n",
    "    neg_id_list.append(str(ag_eval.pk))\n",
    "print('J2:', ','.join(neg_id_list))\n",
    "\n",
    "# -------------------------\n",
    "# POS\n",
    "# -------------------------\n",
    "pos_id_list = list()\n",
    "for meth_id in pos_diff:\n",
    "    ag_eval = AgreementEvaluation.objects.get(reference_method__id=meth_id, evaluator=leo)\n",
    "    pos_id_list.append(str(ag_eval.pk))\n",
    "print('J1:', ','.join(pos_id_list))\n",
    "\n",
    "pos_id_list = list()\n",
    "for meth_id in pos_diff:\n",
    "    ag_eval = AgreementEvaluation.objects.get(reference_method__id=meth_id, evaluator=anto)\n",
    "    pos_id_list.append(str(ag_eval.pk))\n",
    "print('J2:', ','.join(pos_id_list))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### JHotDraw 7.4.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "J1: \n",
      "J2: \n",
      "J1: \n",
      "J2: \n"
     ]
    }
   ],
   "source": [
    "j1 = Judge('leonardo.nole', 'JHotDraw', '7.4.1')\n",
    "j2 = Judge('rossella.linsalata', 'JHotDraw', '7.4.1')\n",
    "\n",
    "j1_evals = j1.two_codes_evaluations\n",
    "j2_evals = j2.two_codes_evaluations\n",
    "\n",
    "neg_diff = j1_evals[0].intersection(j2_evals[1])\n",
    "pos_diff = j1_evals[1].intersection(j2_evals[0])\n",
    "\n",
    "leo = User.objects.get(username='leonardo.nole')\n",
    "anto = User.objects.get(username='rossella.linsalata')\n",
    "\n",
    "# -------------------------\n",
    "# NEG\n",
    "# -------------------------\n",
    "\n",
    "neg_id_list = list()\n",
    "for meth_id in neg_diff:\n",
    "    ag_eval = AgreementEvaluation.objects.get(reference_method__id=meth_id, evaluator=leo)\n",
    "    neg_id_list.append(str(ag_eval.pk))\n",
    "print('J1:', ','.join(neg_id_list))\n",
    "\n",
    "neg_id_list = list()\n",
    "for meth_id in neg_diff:\n",
    "    ag_eval = AgreementEvaluation.objects.get(reference_method__id=meth_id, evaluator=anto)\n",
    "    neg_id_list.append(str(ag_eval.pk))\n",
    "print('J2:', ','.join(neg_id_list))\n",
    "\n",
    "# -------------------------\n",
    "# POS\n",
    "# -------------------------\n",
    "pos_id_list = list()\n",
    "for meth_id in pos_diff:\n",
    "    ag_eval = AgreementEvaluation.objects.get(reference_method__id=meth_id, evaluator=leo)\n",
    "    pos_id_list.append(str(ag_eval.pk))\n",
    "print('J1:', ','.join(pos_id_list))\n",
    "\n",
    "pos_id_list = list()\n",
    "for meth_id in pos_diff:\n",
    "    ag_eval = AgreementEvaluation.objects.get(reference_method__id=meth_id, evaluator=anto)\n",
    "    pos_id_list.append(str(ag_eval.pk))\n",
    "    \n",
    "print('J2:', ','.join(pos_id_list))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TEST: Lexical overlap considering ONLY the Intersection agreement"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MID:  980\n",
      "Jhotdraw (7.4.1) & 2189 & 0.0 & 1.0 & 0.408 & 0.397 & 0.0635 & 0.252 \\\\\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "\n",
    "judges_combinations = (('leonardo.nole', 'rossella.linsalata'),\n",
    "                       ('leonardo.nole', 'rossella.linsalata'),\n",
    "                       ('leonardo.nole', 'antonio.petrone'),\n",
    "                       ('leonardo.nole', 'antonio.petrone'),)\n",
    "\n",
    "CODES_Labels = ('NC', 'DK', 'CO')\n",
    "from collections import defaultdict\n",
    "stats_results = defaultdict(list)\n",
    "\n",
    "for pno, project in enumerate(projects):\n",
    "    \n",
    "    if not pno == 1:\n",
    "        continue\n",
    "\n",
    "    # Get Methods\n",
    "    code_methods = project.code_methods.all()\n",
    "\n",
    "    # Populate the Doc Collection\n",
    "    document_collection = list()\n",
    "    method_ids_map = dict()  # Map (dict) to store the association method.pk --> Row index in Tfidf Matrix\n",
    "    for mno, method in enumerate(code_methods):\n",
    "        clexicon_info = method.lexical_info\n",
    "        document_collection.append(clexicon_info.normalized_comment)\n",
    "        document_collection.append(clexicon_info.normalized_code)\n",
    "        method_ids_map[method.id] = mno * 2\n",
    "\n",
    "    vectorizer = TfidfVectorizer(input='content', sublinear_tf=True, lowercase=False)\n",
    "    tfidf_values = vectorizer.fit_transform(document_collection)\n",
    "\n",
    "    j1_usrname, j2_usrname = judges_combinations[pno]\n",
    "    j1 = Judge(j1_usrname, project.name, project.version)\n",
    "    j2 = Judge(j2_usrname, project.name, project.version)\n",
    "    \n",
    "    j1_evals = j1.three_codes_evaluations\n",
    "    j2_evals = j2.three_codes_evaluations\n",
    "    \n",
    "    project_stats = list()\n",
    "    method_ids = list()\n",
    "    for code in range(3):\n",
    "        j1_evals_code = j1_evals[code]\n",
    "        j2_evals_code = j2_evals[code]\n",
    "        \n",
    "        method_ids.extend(j1_evals_code.intersection(j2_evals_code))\n",
    "        \n",
    "    cosine_sim_vals = list()\n",
    "    for mid in method_ids:\n",
    "        i = method_ids_map[mid]\n",
    "        assert i % 2 == 0, print(i, mid)\n",
    "        dotprod = tfidf_values[i].dot(tfidf_values[i+1].T)[0,0]\n",
    "        cosine_sim_vals.append(dotprod)\n",
    "        if dotprod == 1.0:\n",
    "            print('MID: ', mid)\n",
    "    \n",
    "    vals = np.array(cosine_sim_vals)\n",
    "    print('{proj} ({ver}) & {total} & {min:.3} & {max:.3} & {median:.3} & {mean:.3} & {variance:.3} & {devstd:.3} \\\\\\\\'.format(\n",
    "                                                                                 proj = project.name.title(), \n",
    "                                                                                 ver=project.version,\n",
    "                                                                                 total=vals.size, \n",
    "                                                                                 min=vals.min(), \n",
    "                                                                                 max=vals.max(), \n",
    "                                                                                 median=median(vals), \n",
    "                                                                                 mean=vals.mean(), \n",
    "                                                                                 variance=var(vals), \n",
    "                                                                                 devstd=std(vals)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TEST: Lexical overlap considering ONLY the Methods were Judges Did not Agree on their Coherence Value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Coffeemaker (1.0) & 2 & 0.628 & 0.703 & 0.665 & 0.665 & 0.0014 & 0.0375 \\\\\n",
      "Jhotdraw (7.4.1) & 504 & 0.0 & 0.927 & 0.396 & 0.397 & 0.0612 & 0.247 \\\\\n",
      "Jfreechart (0.6.0) & 14 & 0.0 & 0.582 & 0.266 & 0.251 & 0.0553 & 0.235 \\\\\n",
      "Jfreechart (0.7.1) & 11 & 0.0 & 0.258 & 0.0 & 0.0792 & 0.0112 & 0.106 \\\\\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "\n",
    "judges_combinations = (('leonardo.nole', 'rossella.linsalata'),\n",
    "                       ('leonardo.nole', 'rossella.linsalata'),\n",
    "                       ('leonardo.nole', 'antonio.petrone'),\n",
    "                       ('leonardo.nole', 'antonio.petrone'),)\n",
    "\n",
    "CODES_Labels = ('NC', 'DK', 'CO')\n",
    "from collections import defaultdict\n",
    "stats_results = defaultdict(list)\n",
    "\n",
    "for pno, project in enumerate(projects):\n",
    "\n",
    "    # Get Methods\n",
    "    code_methods = project.code_methods.all()\n",
    "\n",
    "    # Populate the Doc Collection\n",
    "    document_collection = list()\n",
    "    method_ids_map = dict()  # Map (dict) to store the association method.pk --> Row index in Tfidf Matrix\n",
    "    for mno, method in enumerate(code_methods):\n",
    "        clexicon_info = method.lexical_info\n",
    "        document_collection.append(clexicon_info.normalized_comment)\n",
    "        document_collection.append(clexicon_info.normalized_code)\n",
    "        method_ids_map[method.id] = mno * 2\n",
    "\n",
    "    vectorizer = TfidfVectorizer(input='content', sublinear_tf=True, lowercase=False)\n",
    "    tfidf_values = vectorizer.fit_transform(document_collection)\n",
    "\n",
    "    j1_usrname, j2_usrname = judges_combinations[pno]\n",
    "    j1 = Judge(j1_usrname, project.name, project.version)\n",
    "    j2 = Judge(j2_usrname, project.name, project.version)\n",
    "    \n",
    "    j1_evals = j1.three_codes_evaluations\n",
    "    j2_evals = j2.three_codes_evaluations\n",
    "    \n",
    "    project_stats = list()\n",
    "    method_ids = list()\n",
    "    for code in range(3):\n",
    "        j1_evals_code = j1_evals[code]\n",
    "        j2_evals_code = j2_evals[code]\n",
    "        \n",
    "        method_ids.extend(j1_evals_code.intersection(j2_evals_code))\n",
    "        \n",
    "    cosine_sim_vals = list()\n",
    "    for mid in method_ids_map:\n",
    "        if not mid in method_ids:\n",
    "            i = method_ids_map[mid]\n",
    "            cosine_sim_vals.append(tfidf_values[i].dot(tfidf_values[i+1].T)[0,0])\n",
    "    \n",
    "    vals = np.array(cosine_sim_vals)\n",
    "    print('{proj} ({ver}) & {total} & {min:.3} & {max:.3} & {median:.3} & {mean:.3} & {variance:.3} & {devstd:.3} \\\\\\\\'.format(\n",
    "                                                                                 proj = project.name.title(), \n",
    "                                                                                 ver=project.version,\n",
    "                                                                                 total=vals.size, \n",
    "                                                                                 min=vals.min(), \n",
    "                                                                                 max=vals.max(), \n",
    "                                                                                 median=median(vals), \n",
    "                                                                                 mean=vals.mean(), \n",
    "                                                                                 variance=var(vals), \n",
    "                                                                                 devstd=std(vals)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
